"""
auth:xgt-python
datetime:2021/11/7
线程爬虫
"""
import requests
import parsel  # 数据解析模块
import re
import time
import concurrent.futures

url = 'https://fabiaoqing.com/biaoqing/lists/page/{page}.html'


def change_title(name):
    """替换的特殊字符"""
    mode = re.compile(r'[\\\/\:\*\?\<\>\"\|]')
    new_name = re.sub(mode, '_', name)
    return new_name


def get_response(html_url):
    """发送请求的函数"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36'
    }
    response = requests.get(url=url, headers=headers)
    return response


def save(img_url, img_title, suffix):
    try:
        img_content = get_response(img_url).content
        with open('img\\' + img_title + '.' + suffix, mode='wb') as file:
            file.write(img_content)
            print('正在保存:', img_title)
    except:
        pass


def main(html_url):
    html_data = get_response(html_url).text
    selector = parsel.Selector(html_data)
    href = selector.css('#bqb div.ui.segment.imghover img::attr(data-original)').getall()
    names = selector.css('#bqb div.ui.segment.imghover img::attr(title)').getall()
    #   从列表里面  把元素一一提取出来
    # 遍历
    for index in zip(href, names):
        img_url = index[0]
        img_title = change_title(index[1])
        suffix = img_url.split('.')[-1]  # 图片后缀名字
        print(img_title, suffix, img_url, sep=" | ")
        save(img_url, img_title, suffix)


if __name__ == '__main__':
    start_time = time.time()
    exe = concurrent.futures.ThreadPoolExecutor(max_workers=7)
    for page in range(1, 20):
        url = f'https://fabiaoqing.com/biaoqing/lists/page/{page}.html'
        exe.submit(main, url)
    exe.shutdown()
    end_time = time.time()
    use_time = end_time - start_time
    print('总计耗时：', use_time)
